# encoding:utf-8
from __future__ import division
import time
import datetime
import requests
from constant import constant_define
import re
from dao.MySqlDAL import MySqlDAL

'''
备用

http://hq.sinajs.cn/list=sh601003,sh601001


雪球
http://xueqiu.com/v4/stock/quote.json?code=SH601988,SH600000


'''


class StockPriceBackupCrawler(object):
    __target = ""
    __page_size = 600
    __sqlUtil = MySqlDAL()
    __tablename = 'tb_stock_price'

    __headers = {
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'User-Agent': constant_define.user_agent,
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6'
    }


    # 设置目标网址
    def set_target(self, target):
        self.__target = target


    # 将股票代码分组，每组最多900个 [[],[],[]...]
    def get_stock_list(self):
        query_sql = r'SELECT stock_id FROM tb_stock_info'
        res_dict = self.__sqlUtil.get_dimensions_rows(query_sql)
        params = list()
        slen = len(res_dict)
        count = slen//self.__page_size
        for i in xrange(0,count):
            param = res_dict[self.__page_size*i:self.__page_size*(i+1)]
            params.append(param)

        if slen%self.__page_size > 0 :
            param = res_dict[self.__page_size*count:]
            params.append(param)

        return params

    # 处理请求返回json
    def get_request(self,req_url):
        req = requests.get(req_url,headers=self.__headers)
        str_partern = r'hq_str_(\w{8})="(.*?)"'
        rp = re.compile(str_partern)
        tmp_json = dict()
        for row in rp.findall(req.text):
            tmp_json[str(row[0]).upper()] = row[1]
        return tmp_json


    # 获取并设置网页内容
    def get_stock_info_json(self):
        stock_group = self.get_stock_list()
        stock_json = dict()
        for item in stock_group:
            param = ''
            for row in item:
                param += str(row['stock_id']).lower()+','
            param = param[:len(param)-1]
            stock_json.update(self.get_request(self.__target+param))
            # 防止频繁访问网站
            time.sleep(1)
        return stock_json

    # 获取股票流通市值
    def get_market_value(self):
        res_list = self.__sqlUtil.get_dimensions_rows('SELECT stock_id,market_value FROM tb_stock_info')
        stock_list = dict()
        for item in res_list:
            tmp = {item['stock_id']:item['market_value']}
            stock_list.update(tmp)
        return stock_list

    # 写db
    def writeToDB(self):
        mv_dict = self.get_market_value()
        data_keys = ['stock_id','price','percent','open','updown','yestclose','volume','turnover','turnover_rate','date','update_time']
        data_source = []
        for key,value in self.get_stock_info_json().iteritems():
            tmp = list()
            tmp_str = str(value).split(',')
            if len(tmp_str) < 9:
                continue
            tmp.append(key)
            tmp.append(tmp_str[3])
            tmp.append((float(tmp_str[3])-float(tmp_str[2]))/float(tmp_str[2]))
            tmp.append(tmp_str[1])
            tmp.append(float(tmp_str[3])-float(tmp_str[2]))
            tmp.append(tmp_str[2])
            tmp.append(tmp_str[8])
            tmp.append(tmp_str[9])
            if mv_dict[tmp[0]] != 0:
                tmp.append(float(tmp_str[9])/(mv_dict[tmp[0]]*10**8))
            else:
                tmp.append(0)
            tmp.append(datetime.date.today())
            tmp.append(time.time())
            data_source.append(tmp)

        return self.__sqlUtil.insert_many_batch(data_keys,data_source,self.__tablename,50)


    # 更新 db
    def updateDB(self):
        mv_dict = self.get_market_value()
        data_keys = ['price','percent','open','updown','yestclose','volume','turnover','turnover_rate','update_time']
        data_source = []
        filter_collection_key = ['stock_id','date']
        for key,value in self.get_stock_info_json().iteritems():
            tmp = list()
            tmp_str = str(value).split(',')
            if len(tmp_str) < 9:
                continue

            tmp.append(tmp_str[3])
            tmp.append((float(tmp_str[3])-float(tmp_str[2]))/float(tmp_str[2]))
            tmp.append(tmp_str[1])
            tmp.append(float(tmp_str[3])-float(tmp_str[2]))
            tmp.append(tmp_str[2])
            tmp.append(tmp_str[8])
            tmp.append(tmp_str[9])
            if mv_dict[tmp[0]] != 0:
                tmp.append(float(tmp_str[9])/(mv_dict[tmp[0]]*10**8))
            else:
                tmp.append(0)
            tmp.append(time.time())
            tmp.append(key)
            tmp.append(datetime.date.today())
            data_source.append(tmp)

        return self.__sqlUtil.update_many_batch(data_keys,data_source,self.__tablename,filter_collection_key,50)


